{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/hhyhhyhy/anaconda3/lib/python3.5/site-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "import scipy.stats as sps\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import warnings\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "warnings.filterwarnings(action='ignore')\n",
    "\n",
    "path = '../data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_basic = pd.read_pickle(path+'train_basic.pkl')\n",
    "train_lda = pd.read_pickle(path+'train_lda_tfidf.pkl')\n",
    "train_nmf = pd.read_pickle(path+'train_nmf_tfidf.pkl')\n",
    "train_svd = pd.read_pickle(path+'train_svd_tfidf.pkl')\n",
    "train_pos = pd.read_pickle(path+'train_pos.pkl')\n",
    "train_w2v = pd.read_pickle(path+'train_w2v.pkl')\n",
    "\n",
    "valid_basic = pd.read_pickle(path+'valid_basic.pkl')\n",
    "valid_lda = pd.read_pickle(path+'valid_lda_tfidf.pkl')\n",
    "valid_nmf = pd.read_pickle(path+'valid_nmf_tfidf.pkl')\n",
    "valid_svd = pd.read_pickle(path+'valid_svd_tfidf.pkl')\n",
    "valid_pos = pd.read_pickle(path+'valid_pos.pkl')\n",
    "valid_w2v = pd.read_pickle(path+'valid_w2v.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y=pd.read_pickle(path+'train.pkl')['label']\n",
    "y_valid = pd.read_pickle(path+'valid.pkl')['label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(24736, 136) (3092, 136)\n"
     ]
    }
   ],
   "source": [
    "X_train=np.hstack([train_basic,\n",
    "                   train_lda,\n",
    "                   train_nmf,\n",
    "                   train_svd,\n",
    "                   train_pos,\n",
    "                   train_w2v])\n",
    "\n",
    "X_valid = np.hstack([valid_basic,\n",
    "                     valid_lda,\n",
    "                     valid_nmf,\n",
    "                     valid_svd,\n",
    "                     valid_pos,\n",
    "                     valid_w2v])\n",
    "\n",
    "print(X_train.shape,X_valid.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "params={\n",
    "    'max_depth':8,\n",
    "    'nthread':18,\n",
    "    'eta':0.03,\n",
    "    'eval_metric':['error','logloss'],\n",
    "    #'eval_metric':['logloss','error'],\n",
    "    'objective':'binary:logistic',\n",
    "    'subsample':0.7,\n",
    "    'colsample_bytree':0.5,\n",
    "    'silent':1,\n",
    "    'seed':1123,\n",
    "    'min_child_weight':10\n",
    "    #'scale_pos_weight':0.5\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dtrain=xgb.DMatrix(X_train,y)\n",
    "dtest=xgb.DMatrix(X_valid,y_valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tTrain-error:0.275388\tTrain-logloss:0.685178\tTest-error:0.308862\tTest-logloss:0.686413\n",
      "Multiple eval metrics have been passed: 'Test-logloss' will be used for early stopping.\n",
      "\n",
      "Will train until Test-logloss hasn't improved in 20 rounds.\n",
      "[20]\tTrain-error:0.169065\tTrain-logloss:0.557626\tTest-error:0.218629\tTest-logloss:0.581535\n",
      "[40]\tTrain-error:0.153946\tTrain-logloss:0.485595\tTest-error:0.20925\tTest-logloss:0.527953\n",
      "[60]\tTrain-error:0.142788\tTrain-logloss:0.43489\tTest-error:0.206662\tTest-logloss:0.493529\n",
      "[80]\tTrain-error:0.131347\tTrain-logloss:0.39789\tTest-error:0.202458\tTest-logloss:0.470616\n",
      "[100]\tTrain-error:0.120351\tTrain-logloss:0.370026\tTest-error:0.199547\tTest-logloss:0.45477\n",
      "[120]\tTrain-error:0.112023\tTrain-logloss:0.347664\tTest-error:0.198254\tTest-logloss:0.443663\n",
      "[140]\tTrain-error:0.105029\tTrain-logloss:0.329314\tTest-error:0.194049\tTest-logloss:0.435681\n",
      "[160]\tTrain-error:0.09751\tTrain-logloss:0.312588\tTest-error:0.192109\tTest-logloss:0.428671\n",
      "[180]\tTrain-error:0.09092\tTrain-logloss:0.298465\tTest-error:0.191785\tTest-logloss:0.423496\n",
      "[200]\tTrain-error:0.083845\tTrain-logloss:0.285758\tTest-error:0.189198\tTest-logloss:0.419459\n",
      "[220]\tTrain-error:0.078226\tTrain-logloss:0.274322\tTest-error:0.188228\tTest-logloss:0.415539\n",
      "[240]\tTrain-error:0.073213\tTrain-logloss:0.264632\tTest-error:0.186934\tTest-logloss:0.412744\n",
      "[260]\tTrain-error:0.069211\tTrain-logloss:0.255805\tTest-error:0.185964\tTest-logloss:0.41026\n",
      "[280]\tTrain-error:0.064885\tTrain-logloss:0.247086\tTest-error:0.187257\tTest-logloss:0.408585\n",
      "[300]\tTrain-error:0.060277\tTrain-logloss:0.239084\tTest-error:0.187257\tTest-logloss:0.406429\n",
      "[320]\tTrain-error:0.056476\tTrain-logloss:0.231963\tTest-error:0.186287\tTest-logloss:0.404908\n",
      "[340]\tTrain-error:0.052434\tTrain-logloss:0.224598\tTest-error:0.184023\tTest-logloss:0.4028\n",
      "[360]\tTrain-error:0.04928\tTrain-logloss:0.21823\tTest-error:0.183053\tTest-logloss:0.401591\n",
      "[380]\tTrain-error:0.046734\tTrain-logloss:0.212389\tTest-error:0.183053\tTest-logloss:0.40039\n",
      "[400]\tTrain-error:0.043014\tTrain-logloss:0.206303\tTest-error:0.182083\tTest-logloss:0.399114\n",
      "[420]\tTrain-error:0.041235\tTrain-logloss:0.200984\tTest-error:0.182083\tTest-logloss:0.397802\n",
      "[440]\tTrain-error:0.038648\tTrain-logloss:0.195966\tTest-error:0.181436\tTest-logloss:0.397466\n",
      "[460]\tTrain-error:0.036425\tTrain-logloss:0.190861\tTest-error:0.180142\tTest-logloss:0.396452\n",
      "[480]\tTrain-error:0.034201\tTrain-logloss:0.186771\tTest-error:0.180789\tTest-logloss:0.395556\n",
      "[500]\tTrain-error:0.031533\tTrain-logloss:0.181964\tTest-error:0.181436\tTest-logloss:0.395348\n",
      "[520]\tTrain-error:0.029673\tTrain-logloss:0.177885\tTest-error:0.182083\tTest-logloss:0.394759\n",
      "[540]\tTrain-error:0.028097\tTrain-logloss:0.174095\tTest-error:0.180466\tTest-logloss:0.394167\n",
      "[560]\tTrain-error:0.026843\tTrain-logloss:0.16967\tTest-error:0.177878\tTest-logloss:0.393012\n",
      "[580]\tTrain-error:0.025186\tTrain-logloss:0.166115\tTest-error:0.178849\tTest-logloss:0.392329\n",
      "[600]\tTrain-error:0.024135\tTrain-logloss:0.162456\tTest-error:0.179495\tTest-logloss:0.391705\n",
      "[620]\tTrain-error:0.023165\tTrain-logloss:0.15954\tTest-error:0.179172\tTest-logloss:0.391234\n",
      "[640]\tTrain-error:0.021467\tTrain-logloss:0.15615\tTest-error:0.179819\tTest-logloss:0.390637\n",
      "[660]\tTrain-error:0.020416\tTrain-logloss:0.152862\tTest-error:0.178202\tTest-logloss:0.389985\n",
      "[680]\tTrain-error:0.019122\tTrain-logloss:0.149518\tTest-error:0.179495\tTest-logloss:0.389927\n",
      "[700]\tTrain-error:0.017747\tTrain-logloss:0.146461\tTest-error:0.178849\tTest-logloss:0.389329\n",
      "[720]\tTrain-error:0.01702\tTrain-logloss:0.143666\tTest-error:0.178525\tTest-logloss:0.388791\n",
      "[740]\tTrain-error:0.016454\tTrain-logloss:0.141061\tTest-error:0.178525\tTest-logloss:0.388413\n",
      "[760]\tTrain-error:0.015322\tTrain-logloss:0.138473\tTest-error:0.178849\tTest-logloss:0.38805\n",
      "[780]\tTrain-error:0.014432\tTrain-logloss:0.135697\tTest-error:0.179495\tTest-logloss:0.387564\n",
      "[800]\tTrain-error:0.013826\tTrain-logloss:0.133196\tTest-error:0.179172\tTest-logloss:0.387342\n",
      "[820]\tTrain-error:0.013179\tTrain-logloss:0.131074\tTest-error:0.179819\tTest-logloss:0.387103\n",
      "[840]\tTrain-error:0.012371\tTrain-logloss:0.128851\tTest-error:0.180466\tTest-logloss:0.386691\n",
      "[860]\tTrain-error:0.011724\tTrain-logloss:0.126626\tTest-error:0.178849\tTest-logloss:0.386834\n",
      "Stopping. Best iteration:\n",
      "[841]\tTrain-error:0.01233\tTrain-logloss:0.128728\tTest-error:0.179495\tTest-logloss:0.386662\n",
      "\n"
     ]
    }
   ],
   "source": [
    "clf=xgb.train(params,dtrain,\n",
    "              num_boost_round=1000,\n",
    "              early_stopping_rounds=20,\n",
    "              evals=[(dtrain,'Train'),(dtest,'Test')],\n",
    "              verbose_eval=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "valid_pred = clf.predict(dtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "xgb model the accuracy is : 82.18%\n"
     ]
    }
   ],
   "source": [
    "valid_pred = clf.predict(dtest)\n",
    "y_v = (valid_pred+0.5).astype(int)\n",
    "acc =  accuracy_score(y_valid,y_v)\n",
    "\n",
    "print('xgb model the accuracy is : {}%'.format(round(acc* 100,2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#analysis the dev\n",
    "test_basic = pd.read_pickle(path+'dev_basic.pkl')\n",
    "test_lda = pd.read_pickle(path+'dev_lda_tfidf.pkl')\n",
    "test_nmf = pd.read_pickle(path+'dev_nmf_tfidf.pkl')\n",
    "test_svd = pd.read_pickle(path+'dev_svd_tfidf.pkl')\n",
    "test_pos = pd.read_pickle(path+'dev_pos.pkl')\n",
    "test_w2v = pd.read_pickle(path+'dev_w2v.pkl')\n",
    "\n",
    "\n",
    "X_test = np.hstack([test_basic,\n",
    "                    test_lda,\n",
    "                    test_nmf,\n",
    "                    test_svd,\n",
    "                    test_pos,\n",
    "                    test_w2v])\n",
    "\n",
    "y_test = pd.read_pickle(path+'dev.pkl')['label']\n",
    "dtest=xgb.DMatrix(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
